#include "stdafx.h"

#pragma warning( disable : 4786 )

#include "textprocess.h"


#include "math.h"
#include <vector>
#include <set>
//#include <io.h>
#include <map>
#include <fstream>
#include <algorithm>

using namespace std;

HRESULT CTextProcess::ProcessCorpus(char* chTrainDir, char* chTestDir, char* chDataDir, char* chTrainDocId2CatId, char* chTestDocList )
{
	/**********************************************/
	// save the stopword to vector stopword
    vector<string> stopword;
    ifstream stopfile("stopword.txt");
    if (!stopfile)
    {
        cerr << "error: unable to open input file: " << stopfile << endl;
        return -1;
    }
    string tempstr;
    while (getline(stopfile, tempstr))
    {
        stopword.push_back(tempstr);
    }
	cout << "Stopword process has been done." << endl;
	/**********************************************/

    map<int, string> Keywords; // features

	/**********************************************/
	// open the train file
	vector<string> traindoclist;
	ifstream infile("train.doc.list");
	string traindoc;
	while (getline(infile, traindoc))
	{
		traindoclist.push_back(traindoc);
	}
	infile.close();
	cout << "Traindoc reading process is done." << endl;
	/**********************************************/

	ofstream tfvector("tf_train_vector.txt");
	int KeyID = 0;
	/**********************************************/
	for (vector<string>::size_type i = 0; i < traindoclist.size(); i++)
	{
		map<int, int> DocTF;
		string path = "train\\" + string("\\") + traindoclist[i];
		infile.open(path);
		string wordline;
		while(getline(infile, wordline))
		{
			string str;
			for (vector<int>::size_type i = 0; i < wordline.size(); i++)
			{
				if (wordline[i] != ' ' && wordline[i] != '-' && wordline[i] != '!' && wordline[i] != ',' && wordline[i] != '.' && wordline[i] != '?' 
					&& wordline[i] != ':' && !(wordline[i] >= '0' && wordline[i] < '9'))
				{
					str += wordline[i];
				}
				else if (str.size() > 0)
				{
					bool flag = true;
					for (map<int, string>::size_type itr = 0; itr < KeyID; itr++)
					{
						if (str == Keywords[itr])
						{
							DocTF[itr]++;
							flag = false;
							break;
						}
					}
					if (find(stopword.begin(), stopword.end(), str) == stopword.end() && flag)
					{    
						Keywords[KeyID++] = str;
						DocTF[KeyID - 1] = 1;
					}
					str.clear();
				}
			}
		}

		bool isfirst = true;
		tfvector << traindoclist[i] << " ";
		for (int i = 0; i < KeyID; i++)
		{
			if (DocTF[i] > 0)
			{
				if (isfirst)
				{
					tfvector << i << ":" << DocTF[i];
					isfirst = false;
				} 
				else
				{
					tfvector << " " << i << ":" << DocTF[i];
				}
			}
		}

		infile.close();
	}
	cout << "Feature choosing is done." << endl;
	cout << "KeyID = " << KeyID << endl;
	/**********************************************/

	
	/**********************************************/
	// read the test file
	vector<string> CheckDocList;
	ifstream checkfile("check.doc.list");
	if (!checkfile)
	{
		cerr << "error: unable to open input file: " << checkfile << endl;
		return -1;
	}
	string checklist;
	while (getline(checkfile, checklist))
	{
		CheckDocList.push_back(checklist);
	}
	cout << "Read checkdoc file is done." << endl;
	/**********************************************/

	ofstream tftestfile("tf_test_vector.txt");

	for (vector<string>::size_type i = 0; i < CheckDocList.size(); i++)
	{
		map<int ,int> TestTF;
		string path = "test\\" + string("\\") + CheckDocList[i];
		cout << path << endl;
		infile.open(path);
		string wordline;
		while (getline(infile, wordline))
		{
			string str;
			for (vector<int>::size_type i = 0; i < wordline.size(); i++)
			{
				if (wordline[i] != ' ' && wordline[i] != '-' && wordline[i] != '!' && wordline[i] != ',' && wordline[i] != '.' && wordline[i] != '?' 
					&& wordline[i] != ':' && !(wordline[i] >= '0' && wordline[i] < '9'))
				{
					str += wordline[i];
				}
				else if (str.size() > 0)
				{
					for (map<int, string>::size_type itr = 0; itr < KeyID; itr++)
					{
						if (str == Keywords[itr])
						{
							TestTF[itr]++;
							break;
						}
					}
					str.clear();
				}
			}
		}

		bool isfirst = true;
		tftestfile << CheckDocList[i] << " ";
		for (int i = 0; i < KeyID; i++)
		{
			if (TestTF[i] > 0)
			{
				if (isfirst)
				{
					tftestfile << i << ":" << TestTF[i];
					isfirst = false;
				} 
				else
				{
					tftestfile << " " << i << ":" << TestTF[i];
				}
			}
		}
		infile.close();
	}

	cout << "Done." << endl;
    return 0;
}